﻿using System;
using System.Collections.Generic;
using System.Web;
using System.Text;
using Gapi.Search;
using System.Net;
using System.IO;
using System.Collections;
using System.Text.RegularExpressions;


namespace WebTempGen
{
   
    class WebSiteRetriever
    {
        public List<WSRPage> GetWebsites(string searchTerm, int numPages, bool ignoreLoadTime)
        {
            //http://www.codeplex.com/GAPIdotNET
            int index = 0;
            List<WSRPage> lWSRPage = new List<WSRPage>();
            while (lWSRPage.Count < numPages)
            {
              SearchResults searchResultTemp = Searcher.Search(SearchType.Web, searchTerm, index);
              foreach (SearchResult result in searchResultTemp.Items)
              {
                  string URL = result.Url;

                  Hashtable content = getContent(URL);
                  lWSRPage.Add(new WSRPage(URL, content, loadTime(URL, ignoreLoadTime), getAdditionalPages(content[UserAgent.IE].ToString(),URL)));
                  if (lWSRPage.Count == numPages)
                      break; 
              }
              index++;
            }

            return lWSRPage;
        }


        private List<string> getAdditionalPages(string content, string baseURL)
        {
            List<string> ap = new List<string>();

            Regex additionalPages = new Regex(
                  "(?<=<link\\s*rel=('|\")stylesheet('|\").*href=('|\"))[^'\"]*"+
                  "(?=('|\"))",
                RegexOptions.ExplicitCapture
                | RegexOptions.CultureInvariant
                | RegexOptions.IgnorePatternWhitespace
                | RegexOptions.Compiled
                );

            MatchCollection mc = additionalPages.Matches(content);
            foreach (Match url in mc)
            {
                string URL = string.Empty;
                if (!url.Value.StartsWith("http"))
                    URL = baseURL.TrimEnd('/') + '/' + url.Value.TrimStart('/');
                else
                    URL = url.Value;

                string additionalContent = getContent(URL.Replace("amp;", ""), true);
                if (!additionalContent.Equals(string.Empty))
                    ap.Add(additionalContent);
            }


            return ap;
        }

        private int loadTime(string URL, bool ignoreLoadTime)
        {
            if (!ignoreLoadTime)
            {
                long total = 0;
                int numSamples = 10;
                for (int i = 1; i <= numSamples; i++)
                {
                    long startTicks = DateTime.Now.Ticks;
                    // prepare the web page we will be asking for
                    HttpWebRequest request = (HttpWebRequest)
                        WebRequest.Create(URL);

                    // execute the request
                    HttpWebResponse response = (HttpWebResponse)
                        request.GetResponse();
                    long endTicks = DateTime.Now.Ticks;

                    long diff = endTicks - startTicks;
                    total += diff;
                }
                long averageTicks = total / numSamples;

                TimeSpan ts = new TimeSpan(averageTicks);
                return (int)ts.TotalMilliseconds;
            }
            else
                return 1000;
        }

        private Hashtable getContent(string URL)
        {
            //http://www.csharp-station.com/HowTo/HttpWebFetch.aspx
            // used to build entire input
            StringBuilder sb = new StringBuilder();

            // used on each read operation
            byte[] buf = new byte[8192];

            // prepare the web page we will be asking for
            HttpWebRequest request = (HttpWebRequest)
                WebRequest.Create(URL);

            Hashtable content = new Hashtable();
            foreach (UserAgent ua in Enum.GetValues(typeof(UserAgent)))
            {
                request.UserAgent = GetUAString(ua);
                content[ua] = getContentPerUA(sb, buf, request);
            }       

            
            return content;
        }

        private string getContent(string URL, bool single)
        {
            //http://www.csharp-station.com/HowTo/HttpWebFetch.aspx
            // used to build entire input
            StringBuilder sb = new StringBuilder();

            // used on each read operation
            byte[] buf = new byte[8192];

            // prepare the web page we will be asking for
            HttpWebRequest request = (HttpWebRequest)
                WebRequest.Create(URL);

            request.UserAgent = GetUAString(UserAgent.IE);
            return getContentPerUA(sb, buf, request);
        }

        private static string getContentPerUA(StringBuilder sb, byte[] buf, HttpWebRequest request)
        {
            try
            {
                // execute the request
                HttpWebResponse response = (HttpWebResponse)
                    request.GetResponse();

                // we will read data via the response stream
                Stream resStream = response.GetResponseStream();

                string tempString = null;
                int count = 0;

                do
                {
                    // fill the buffer with data
                    count = resStream.Read(buf, 0, buf.Length);

                    // make sure we read some data
                    if (count != 0)
                    {
                        // translate from bytes to ASCII text
                        tempString = Encoding.ASCII.GetString(buf, 0, count);

                        // continue building the string
                        sb.Append(tempString);
                    }
                }
                while (count > 0); // any more data to read?
                return sb.ToString();
            }
            catch
            {
                return string.Empty;
            }
        }

        public class WSRPage
        {
            private string url;
            private int loadTime;
            private Hashtable content;
            private List<string> additionalPages;

            public List<string> AdditionalPages
            {
                get { return additionalPages; }
                set { additionalPages = value; }
            }

            public string URL
            {
                get { return this.url; }
                set { this.url = value; }
            }

            public Hashtable Content
            {
                get { return this.content; }
                set { this.content = value; }
            }

            public int LoadTime
            {
                get { return loadTime; }
                set { this.loadTime = value; }
            }

            public WSRPage(string url, Hashtable content, int loadTime, List<string> additionalPages)
            {
                this.URL = url;
                this.Content = content;
                this.LoadTime = loadTime;
                this.AdditionalPages = additionalPages;
            }
        }

        private string GetUAString(UserAgent ua)
        {
            switch (ua)
            {
                case UserAgent.IE:
                    return "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.30)";

                case UserAgent.Safari:
                    return "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; de-de) AppleWebKit/125.5.7 (KHTML, like Gecko) Safari/125.12";

                case UserAgent.Firefox:
                    return "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.0.4) Gecko/2008102920 Firefox/3.0.4 (.NET CLR 3.5.30729)";

                case UserAgent.Opera:
                    return "Opera/9.00 (Windows NT 5.1; U; en)";

                default:
                    return string.Empty;
            }
        }
    }

    public enum UserAgent
    {
        IE,
        Firefox,
        Safari,
        Opera
    }
}
